package com.chance.cc.crawler.development.bootstrap.weibo;

import com.alibaba.fastjson.JSON;
import com.chance.cc.crawler.core.CrawlerEnum;
import com.chance.cc.crawler.core.downloader.HttpConfig;
import com.chance.cc.crawler.core.downloader.proxy.Proxy;
import com.chance.cc.crawler.core.filter.FilterUtils;
import com.chance.cc.crawler.core.record.CrawlerRecord;
import com.chance.cc.crawler.core.record.CrawlerRequestRecord;
import com.chance.cc.crawler.development.utils.RedisReader;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.lang3.StringUtils;

import java.io.BufferedReader;
import java.io.FileInputStream;
import java.io.InputStreamReader;
import java.util.Arrays;
import java.util.List;

import static com.chance.cc.crawler.core.CrawlerEnum.CrawlerRecordFilter.dateRange;
import static com.chance.cc.crawler.core.CrawlerEnum.CrawlerRecordFilter.keyOrDateRange;
import static com.chance.cc.crawler.core.CrawlerEnum.CrawlerRequestType.turnPageItem;

/**
 * @Author Zhao.Hhuan
 * @Date Create in 2021/1/8 15:00
 * @Description
 *      微博帖子地址列表
 **/
public class WeiboPostUrlListStart {
    public static final String domainId = "weibo";
    private static final List<String> urlList = Arrays.asList("http://weibo.com/5879077231/JBbRmAo4P","http://weibo.com/5213815179/JBbRjEaVr");

    private static Proxy proxy = new Proxy();
    static {
        //代理配置
        proxy.setHost("http-dyn.abuyun.com");
        proxy.setPort(9020);
        proxy.setUsername("HL89Q19E86E2987D");
        proxy.setPassword("71F33D94CE5F7BF2");
    }
    HttpConfig httpConfig = HttpConfig.me(domainId);


    public static void main(String[] args) {
//        createItemRecordToRedis();//采集文章、互动量、回复
        createCommentRecordToRedis();//只采集回复
    }

    //将文件中的地址生成record放入redis中
    private static void createItemRecordToRedis(){
        String site = "post";
        String siteBiz = "self_post-simple";
        String path = "C:\\Users\\Administrator\\Desktop\\weibo_post_url_list.txt";
        HttpConfig httpConfig = HttpConfig.me(domainId);
        httpConfig.setCircularRedirectsAllowed(true);
        //push kv first
        RedisReader redisReader = new RedisReader("192.168.1.214",6379,3);
        try {
            BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(new FileInputStream(path), "UTF-8"));
//
            String readLine = null;
            while ((readLine = bufferedReader.readLine()) != null) {
//            for (String readLine : urlList) {
                if(StringUtils.isBlank(readLine)){
                    continue;
                }
                CrawlerRequestRecord requestRecord = CrawlerRequestRecord.builder()
                        .startPageRequest(domainId, turnPageItem)
                        .httpUrl(readLine)
                        .releaseTime(System.currentTimeMillis())
                        .proxy(proxy)
                        .resultLabelTag(CrawlerEnum.CrawlerDataType.article)
                        .resultLabelTag(CrawlerEnum.CrawlerDataType.interaction)
                        .resultLabelTag(CrawlerEnum.CrawlerDataType.comment)
                        .build();
                requestRecord.tagsCreator().bizTags().addDomain(domainId);
                requestRecord.tagsCreator().bizTags().addSite(site);
                requestRecord.tagsCreator().bizTags().addSiteBiz(siteBiz);

                //添加过滤的方式
                CrawlerRecord commentFilter = new CrawlerRequestRecord();
                commentFilter.setFilter(keyOrDateRange);
                commentFilter.addFilterInfo(FilterUtils.redisFilterKeyInfo(StringUtils.joinWith("-","filter",domainId,site,siteBiz,"queue")));
                commentFilter.addFilterInfo(FilterUtils.dateRangeFilterInfo(0,new long[]{1631721600000L,System.currentTimeMillis()}));
                requestRecord.tagsCreator().resultTags().getCategoryTag().addKVTag("comment_filter_record", JSON.toJSONString(commentFilter));

                String requestKey = requestRecord.getRecordKey();
                //确保key值唯一
                String shaKey = DigestUtils.sha1Hex(System.currentTimeMillis()+requestKey);

                redisReader.mapPush("crawler-weibo-post-self_post-simple-queue_map",shaKey, JSON.toJSONString(requestRecord));
                redisReader.listPush("crawler-weibo-post-self_post-simple-queue_list",shaKey);
            }
        } catch (Exception e) {
            e.printStackTrace();
        }
    }

    private static void createCommentRecordToRedis(){
        String site = "post";
        String siteBiz = "self_post-simple";
        String path = "E:\\chance-crawler-development\\crawler-dev-bootstrap\\src\\main\\java\\com\\chance\\cc\\crawler\\development\\bootstrap\\weibo\\weibo_comment.txt";//改成自己本地路径
        //push kv first
        RedisReader redisReader = new RedisReader("192.168.1.211",6379,3);
        try {
            BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(new FileInputStream(path), "UTF-8"));
            String readLine = null;
            while ((readLine = bufferedReader.readLine()) != null) {
                if(StringUtils.isBlank(readLine)){
                    continue;
                }

                String[] split = readLine.split("\t");
                System.out.println(split[0]);
                System.out.println(split[1]);
                String url = "https://weibo.com/aj/v6/comment/big?ajwvr=6&id="+split[1]+"&filter=all&from=singleWeiBo";//mid
                CrawlerRequestRecord requestRecord = CrawlerRequestRecord.builder()
                        .startPageRequest(domainId, turnPageItem)
                        .httpUrl(url)
                        .releaseTime(System.currentTimeMillis())
                        .proxy(proxy)
                        .filter(dateRange)
//                        .addFilterInfo(FilterUtils.redisFilterKeyInfo(StringUtils.joinWith("-","filter",domainId,site,siteBiz,"queue")))
                        .addFilterInfo(FilterUtils.dateRangeFilterInfo(0,new long[]{1629043200000L,System.currentTimeMillis()}))//时间范围
                        .resultLabelTag(CrawlerEnum.CrawlerDataType.comment)
                        .build();
                requestRecord.tagsCreator().bizTags().addDomain(domainId);
                requestRecord.tagsCreator().bizTags().addSite(site);
                requestRecord.tagsCreator().bizTags().addSiteBiz(siteBiz);
                requestRecord.getHttpRequest().addExtra("articleUrl",split[0]);//文章地址

               String requestKey = requestRecord.getRecordKey();
                //确保key值唯一
                String shaKey = DigestUtils.sha1Hex(System.currentTimeMillis()+requestKey);

                redisReader.mapPush("crawler-weibo-post-self_post-simple-queue_map",shaKey, JSON.toJSONString(requestRecord));
                redisReader.listPush("crawler-weibo-post-self_post-simple-queue_list",shaKey);
            }
        } catch (Exception e) {
            e.printStackTrace();
        }
    }

}
